# coding = utf-8
import io
import os
filePath="/home/qwerty/brat-v1.3_Crunchy_Frog/data/projects/bn/adj"
fileList=os.listdir(filePath)

for file in fileList:
    if ".txt" in os.path.join(filePath, file):
        infile = io.open(os.path.join(filePath, file), 'r',encoding='utf-8')
        transit = ""

        ifHeadline = 0
        for line in infile:
            # words = line.rstrip('\n').split(' ')
            period=0
            isChinese=0
            if "<HEADLINE>" in line:
                ifHeadline = 1
            if "</TURN>" in line:
                continue
            if "<TURN>" in line:
                transit += '\n'
                continue
            for ch in line:
                if '\u4e00' <= ch <= '\u9fff':
                    isChinese = 1
                    break
            if isChinese == 0:
                transit += line
            elif ifHeadline == 1:
                transit += line
                ifHeadline = 0
            else:
                for ch in line:
                    if period==1:
                        period=0
                        if ch!='”' and ch!='"':
                            transit += '\n'
                        else:
                            transit += ch
                            transit += '\n'
                            continue
                    if ch != '\n' and ch != ' ' and ch !='\t':
                        transit += ch
                        if ch == '。':
                            period=1



        infile.close()
        onfile = io.open(os.path.join(filePath, file), 'w',encoding='utf-8')
        onfile.write(transit)
        onfile.close()
